﻿using HtmlAgilityPack;

namespace ImageCrawler
{
    public class ImageCrawler
    {
        private readonly string _baseUrl;
        private readonly string[] _keywords;
        private readonly string _savePath;
        private readonly HttpClient _httpClient;
        private readonly HashSet<string> _visitedUrls;
        private readonly HashSet<string> _downloadedImageUrls; // 新增：记录已下载的图片URL
        private int _downloadedCount;

        public ImageCrawler(string baseUrl, string[] keywords, string savePath)
        {
            _baseUrl = baseUrl;
            _keywords = keywords.Select(k => k.Trim().ToLower()).ToArray();
            _savePath = savePath;
            _httpClient = new HttpClient();
            _httpClient.DefaultRequestHeaders.Add("User-Agent",
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
            _visitedUrls = new HashSet<string>();
            _downloadedImageUrls = new HashSet<string>(); // 初始化已下载图片URL集合
            _downloadedCount = 0;
        }

        public async Task StartCrawling()
        {
            // 确保保存目录存在
            Directory.CreateDirectory(_savePath);

            await CrawlPage(_baseUrl);
        }

        private async Task CrawlPage(string url)
        {
            if (_visitedUrls.Contains(url))
                return;

            _visitedUrls.Add(url);

            try
            {
                Console.WriteLine($"正在访问: {url}");

                var html = await _httpClient.GetStringAsync(url);
                var htmlDocument = new HtmlDocument();
                htmlDocument.LoadHtml(html);

                // 提取并下载图片
                await DownloadImages(htmlDocument, url);

                // 提取并继续爬取链接
                await ExtractAndCrawlLinks(htmlDocument, url);
            }
            catch (Exception ex)
            {
                Console.WriteLine($"访问 {url} 时出错: {ex.Message}");
            }
        }

        private async Task DownloadImages(HtmlDocument document, string baseUrl)
        {
            Console.WriteLine($"开始分析页面: {baseUrl}");

            // 调试：输出HTML结构信息
            var html = document.DocumentNode.OuterHtml;
            Console.WriteLine($"HTML长度: {html.Length} 字符");

            // 更全面的图片检测策略
            var imageNodes = GetAllImageNodes(document);

            if (imageNodes == null || imageNodes.Count == 0)
            {
                Console.WriteLine("未找到任何图片元素，尝试深度搜索...");
                await DeepSearchForImages(document, baseUrl);
                return;
            }

            Console.WriteLine($"找到 {imageNodes.Count} 个图片元素");

            foreach (var imgNode in imageNodes)
            {
                // 调试：输出节点信息
                Console.WriteLine($"分析图片节点: {imgNode.XPath}");

                // 尝试获取多种图片属性
                var imageUrl = GetImageUrlFromNode(imgNode);
                if (string.IsNullOrEmpty(imageUrl))
                {
                    Console.WriteLine("无法提取图片URL，跳过");
                    continue;
                }

                Console.WriteLine($"提取到图片URL: {imageUrl}");

                // 放宽域名限制，支持更多图片源
                if (!IsSupportedImageUrl(imageUrl))
                {
                    Console.WriteLine($"不支持的图片URL格式: {imageUrl}");
                    continue;
                }

                imageUrl = GetAbsoluteUrl(imageUrl, baseUrl);

                // 检查是否已经下载过这个URL
                var normalizedUrl = NormalizeImageUrl(imageUrl);
                if (_downloadedImageUrls.Contains(normalizedUrl))
                {
                    Console.WriteLine($"⏭️ 跳过已记录的图片URL: {normalizedUrl}");
                    continue;
                }

                Console.WriteLine($"处理后的图片链接: {imageUrl}");

                var altText = imgNode.GetAttributeValue("alt", "").ToLower();
                var titleText = imgNode.GetAttributeValue("title", "").ToLower();

                if (_keywords.Length == 0 || (_keywords.Length == 1 && string.IsNullOrWhiteSpace(_keywords[0])))
                {
                    await DownloadImage(imageUrl, altText);
                }
                else if (ContainsKeywords(altText) || ContainsKeywords(titleText))
                {
                    await DownloadImage(imageUrl, altText);
                }
            }
        }

        private HtmlNodeCollection GetAllImageNodes(HtmlDocument document)
        {
            // 多种XPath选择器组合
            var xpathQueries = new[]
            {
                "//img",
                "//*[contains(@class, 'image')]//img",
                "//*[@data-image]",
                "//*[contains(@style, 'background-image')]",
                "//*[contains(@class, 'lazy')]",
                "//*[contains(@class, 'thumbnail')]//img",
                "//*[contains(@class, 'gallery')]//img",
                "//*[@data-src]",
                "//*[@data-original]",
                "//*[@data-lazy]",
                "//picture//img",
                "//figure//img",
                "//*[contains(@class, 'photo')]//img",
                "//*[contains(@class, 'pic')]//img"
            };

            foreach (var xpath in xpathQueries)
            {
                var nodes = document.DocumentNode.SelectNodes(xpath);
                if (nodes != null && nodes.Count > 0)
                {
                    Console.WriteLine($"通过XPath '{xpath}' 找到 {nodes.Count} 个元素");
                    return nodes;
                }
            }

            return null;
        }

        private async Task DeepSearchForImages(HtmlDocument document, string baseUrl)
        {
            Console.WriteLine("开始深度搜索图片...");

            var html = document.DocumentNode.OuterHtml;

            // 更全面的正则表达式模式
            var imageUrlPatterns = new[]
            {
                // 标准图片URL
                @"https?://[^\s""'<>]+\.(jpg|jpeg|png|gif|webp|bmp|svg)(?:\?[^\s""'<>]*)?",
                @"//[^\s""'<>]+\.(jpg|jpeg|png|gif|webp|bmp|svg)(?:\?[^\s""'<>]*)?",
                
                // 带参数的图片URL
                @"https?://[^\s""'<>]+/[\w\-]+\.(jpg|jpeg|png|gif|webp|bmp|svg)\??[\w\-=&%]*",
                
                // CDN和图片服务URL
                @"https?://(?:i\.)?imgur\.com/[^\s""'<>]+",
                @"https?://(?:i\.)?pixiv\.net/[^\s""'<>]+",
                @"https?://images\.unsplash\.com/[^\s""'<>]+",
                @"https?://pixiv-image\.pwp\.link/[^\s""'<>]+"
            };

            var foundUrls = new HashSet<string>();

            foreach (var pattern in imageUrlPatterns)
            {
                var matches = System.Text.RegularExpressions.Regex.Matches(html, pattern,
                    System.Text.RegularExpressions.RegexOptions.IgnoreCase);

                Console.WriteLine($"正则模式 '{pattern}' 找到 {matches.Count} 个匹配");

                foreach (System.Text.RegularExpressions.Match match in matches)
                {
                    var imageUrl = match.Value.Trim();
                    if (string.IsNullOrEmpty(imageUrl))
                        continue;

                    if (imageUrl.StartsWith("//"))
                        imageUrl = "https:" + imageUrl;

                    // 去重
                    if (foundUrls.Contains(imageUrl))
                        continue;

                    foundUrls.Add(imageUrl);

                    if (IsSupportedImageUrl(imageUrl))
                    {
                        imageUrl = GetAbsoluteUrl(imageUrl, baseUrl);
                        Console.WriteLine($"通过深度搜索发现图片: {imageUrl}");
                        await DownloadImage(imageUrl, "deep_search");
                    }
                }
            }

            if (foundUrls.Count == 0)
            {
                Console.WriteLine("深度搜索也未找到任何图片URL");
                Console.WriteLine("建议检查：");
                Console.WriteLine("1. 网页是否使用JavaScript动态加载图片");
                Console.WriteLine("2. 是否需要处理登录或认证");
                Console.WriteLine("3. 网页结构是否特别复杂");

                // 输出部分HTML内容用于调试
                var sampleHtml = html.Length > 500 ? html.Substring(0, 500) + "..." : html;
                Console.WriteLine($"HTML预览: {sampleHtml}");
            }
        }

        private string GetImageUrlFromNode(HtmlNode node)
        {
            // 扩展的属性列表
            var attributes = new[] {
                "src", "data-src", "data-original", "data-url", "data-image",
                "data-lazy", "srcset", "data-srcset", "data-hi-res", "data-low-res",
                "data-file", "data-thumb", "data-full", "data-large", "data-medium"
            };

            foreach (var attr in attributes)
            {
                var url = node.GetAttributeValue(attr, "").Trim();
                if (!string.IsNullOrEmpty(url))
                {
                    Console.WriteLine($"从属性 '{attr}' 提取到URL: {url}");

                    // 处理 srcset 属性
                    if (attr == "srcset" || attr == "data-srcset")
                    {
                        var urls = url.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                        if (urls.Length > 0)
                        {
                            // 取第一个（通常是最大的）图片
                            var firstUrl = urls[0].Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)[0];
                            if (!string.IsNullOrEmpty(firstUrl))
                                return firstUrl;
                        }
                    }
                    else
                    {
                        return url;
                    }
                }
            }

            // 检查背景图片
            var style = node.GetAttributeValue("style", "");
            if (!string.IsNullOrEmpty(style))
            {
                var bgMatch = System.Text.RegularExpressions.Regex.Match(style,
                    @"background-image\s*:\s*url\(['""]?([^'""\)]+)['""]?\)",
                    System.Text.RegularExpressions.RegexOptions.IgnoreCase);

                if (bgMatch.Success)
                {
                    var bgUrl = bgMatch.Groups[1].Value.Trim();
                    Console.WriteLine($"从CSS背景提取到URL: {bgUrl}");
                    return bgUrl;
                }
            }

            return string.Empty;
        }

        private bool IsSupportedImageUrl(string url)
        {
            // 大幅放宽限制，支持更多图片源
            var supportedDomains = new[] {
                "pixiv-image.pwp.link", "i.pximg.net", "imgur.com", "images.unsplash.com",
                "pixiv.net", "cdn.discordapp.com", "media.discordapp.net", "i.redd.it",
                "preview.redd.it", "static.flickr.com", "live.staticflickr.com"
            };

            var supportedExtensions = new[] {
                ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg", ".ico"
            };

            var urlLower = url.ToLower();

            // 检查域名
            if (supportedDomains.Any(domain => urlLower.Contains(domain)))
                return true;

            // 检查文件扩展名
            if (supportedExtensions.Any(ext => urlLower.Contains(ext)))
                return true;

            // 检查常见的图片URL模式
            if (urlLower.Contains("/images/") || urlLower.Contains("/img/") ||
                urlLower.Contains("/photos/") || urlLower.Contains("/pics/"))
                return true;

            return false;
        }

        private async Task SearchForImageUrls(HtmlDocument document, string baseUrl)
        {
            // 在HTML文本中搜索图片URL模式
            var html = document.DocumentNode.OuterHtml;
            var imageUrlPatterns = new[]
            {
                @"https?://[^\s""']+\.(jpg|jpeg|png|gif|webp|bmp)(?:\?[^\s""']*)?",
                @"//[^\s""']+\.(jpg|jpeg|png|gif|webp|bmp)(?:\?[^\s""']*)?"
            };

            foreach (var pattern in imageUrlPatterns)
            {
                var matches = System.Text.RegularExpressions.Regex.Matches(html, pattern,
                    System.Text.RegularExpressions.RegexOptions.IgnoreCase);

                foreach (System.Text.RegularExpressions.Match match in matches)
                {
                    var imageUrl = match.Value;
                    if (imageUrl.StartsWith("//"))
                        imageUrl = "https:" + imageUrl;

                    if (IsSupportedImageUrl(imageUrl))
                    {
                        imageUrl = GetAbsoluteUrl(imageUrl, baseUrl);
                        Console.WriteLine($"通过正则匹配发现图片: {imageUrl}");
                        await DownloadImage(imageUrl, "regex_matched");
                    }
                }
            }
        }

        private bool ContainsKeywords(string text)
        {
            return _keywords.Any(keyword =>
                !string.IsNullOrEmpty(keyword) && text.Contains(keyword));
        }

        private string GetAbsoluteUrl(string relativeUrl, string baseUrl)
        {
            if (Uri.IsWellFormedUriString(relativeUrl, UriKind.Absolute))
                return relativeUrl;

            if (relativeUrl.StartsWith("//"))
                return "https:" + relativeUrl;

            if (relativeUrl.StartsWith("/"))
            {
                var uri = new Uri(baseUrl);
                return $"{uri.Scheme}://{uri.Host}{relativeUrl}";
            }

            // 处理相对路径
            var baseUri = new Uri(baseUrl);
            return new Uri(baseUri, relativeUrl).ToString();
        }

        private async Task DownloadImage(string imageUrl, string altText)
        {
            try
            {
                // 标准化URL（去除查询参数和片段）
                var normalizedUrl = NormalizeImageUrl(imageUrl);

                // 检查是否已经下载过这个图片
                if (_downloadedImageUrls.Contains(normalizedUrl))
                {
                    Console.WriteLine($"⏭️ 跳过已下载的图片: {normalizedUrl}");
                    return;
                }

                var response = await _httpClient.GetAsync(imageUrl);
                if (response.IsSuccessStatusCode)
                {
                    var imageData = await response.Content.ReadAsByteArrayAsync();

                    // 检查文件是否已存在（基于内容哈希）
                    var fileHash = ComputeFileHash(imageData);
                    var existingFile = FindExistingFile(fileHash);
                    if (existingFile != null)
                    {
                        Console.WriteLine($"⏭️ 跳过重复文件（已存在）: {existingFile}");
                        _downloadedImageUrls.Add(normalizedUrl); // 仍然记录为已下载
                        return;
                    }

                    // 生成文件名
                    var fileName = GenerateFileName(imageUrl, altText, fileHash);
                    var filePath = Path.Combine(_savePath, fileName);

                    await File.WriteAllBytesAsync(filePath, imageData);

                    // 记录已下载的图片URL
                    _downloadedImageUrls.Add(normalizedUrl);
                    _downloadedCount++;

                    Console.WriteLine($"✓ 下载成功: {fileName} (来自: {GetDomain(imageUrl)})");
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"下载图片失败 {imageUrl}: {ex.Message}");
            }
        }

        // 新增方法：标准化图片URL（去除查询参数和片段）
        private string NormalizeImageUrl(string url)
        {
            try
            {
                var uri = new Uri(url);

                // 构建标准化的URL（只包含协议、主机、路径）
                var normalized = $"{uri.Scheme}://{uri.Host}{uri.AbsolutePath}";

                // 如果是常见的图片CDN，可以进一步处理
                if (uri.Host.Contains("pixiv-image.pwp.link") || uri.Host.Contains("i.pximg.net"))
                {
                    // 对于Pixiv图片，可以基于路径进行标准化
                    var pathParts = uri.AbsolutePath.Split('/');
                    if (pathParts.Length >= 3)
                    {
                        // 提取图片ID部分
                        normalized = $"{uri.Scheme}://{uri.Host}/{pathParts[1]}/{pathParts[2]}";
                    }
                }

                return normalized.ToLower();
            }
            catch
            {
                // 如果URL解析失败，返回原始URL的小写形式
                return url.ToLower();
            }
        }

        // 新增方法：计算文件内容的哈希值
        private string ComputeFileHash(byte[] data)
        {
            using var sha256 = System.Security.Cryptography.SHA256.Create();
            var hashBytes = sha256.ComputeHash(data);
            return BitConverter.ToString(hashBytes).Replace("-", "").ToLower();
        }

        // 新增方法：查找已存在的相同文件
        private string FindExistingFile(string fileHash)
        {
            try
            {
                if (!Directory.Exists(_savePath))
                    return null;

                var files = Directory.GetFiles(_savePath, "*.*", SearchOption.AllDirectories);
                foreach (var file in files)
                {
                    try
                    {
                        var existingData = File.ReadAllBytes(file);
                        var existingHash = ComputeFileHash(existingData);
                        if (existingHash == fileHash)
                            return file;
                    }
                    catch
                    {
                        // 忽略无法读取的文件
                        continue;
                    }
                }
            }
            catch
            {
                // 忽略目录访问错误
            }

            return null;
        }

        // 修改文件名生成方法，包含哈希值
        private string GenerateFileName(string imageUrl, string altText, string fileHash = null)
        {
            var extension = Path.GetExtension(imageUrl.Split('?')[0]);
            if (string.IsNullOrEmpty(extension))
                extension = ".jpg";

            var safeAltText = string.IsNullOrEmpty(altText)
                ? "image"
                : string.Join("_", altText.Split(Path.GetInvalidFileNameChars()));

            // 使用哈希值前8位作为唯一标识
            var hashPart = fileHash?.Substring(0, 8) ?? "unknown";

            return $"{safeAltText}_{hashPart}{extension}";
        }

        private async Task ExtractAndCrawlLinks(HtmlDocument document, string baseUrl)
        {
            var linkNodes = document.DocumentNode.SelectNodes("//a[@href]");
            if (linkNodes == null)
                return;

            var tasks = new List<Task>();
            foreach (var linkNode in linkNodes)
            {
                var href = linkNode.GetAttributeValue("href", "");
                if (string.IsNullOrEmpty(href) || href.StartsWith("#"))
                    continue;

                var absoluteUrl = GetAbsoluteUrl(href, baseUrl);

                // 只爬取同一域名的链接
                if (GetDomain(absoluteUrl) == GetDomain(_baseUrl))
                {
                    tasks.Add(CrawlPage(absoluteUrl));
                }
            }

            await Task.WhenAll(tasks);
        }

        private string GetDomain(string url)
        {
            try
            {
                var uri = new Uri(url);
                return uri.Host;
            }
            catch
            {
                return string.Empty;
            }
        }
    }
}